{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第七步：减小学习率，再次调节n_estimators\n",
    "减小学习率为0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "dpath = './'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "X_train = np.array(train.drop([\"interest_level\"], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练\n",
    "采用前几步调整的参数：max_depth = 5，min_child_weight = 4，subsample = 0.8， colsample_bytree = 0.7，reg_alpha = 1.5，reg_lambda = 0.5，\n",
    "调整learning_rate为0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=200):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print(cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('5_nestimators.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( '5_nestimators.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print(\"logloss of train :\" )\n",
    "    print(logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      train-mlogloss-mean  train-mlogloss-std  test-mlogloss-mean  \\\n",
      "0                1.092618            0.000018            1.092666   \n",
      "1                1.086701            0.000082            1.086797   \n",
      "2                1.080949            0.000101            1.081101   \n",
      "3                1.075306            0.000165            1.075489   \n",
      "4                1.069660            0.000233            1.069886   \n",
      "5                1.064081            0.000214            1.064364   \n",
      "6                1.058652            0.000276            1.058983   \n",
      "7                1.053380            0.000327            1.053758   \n",
      "8                1.048074            0.000295            1.048502   \n",
      "9                1.042915            0.000262            1.043385   \n",
      "10               1.037890            0.000180            1.038395   \n",
      "11               1.032818            0.000106            1.033363   \n",
      "12               1.027850            0.000221            1.028450   \n",
      "13               1.022997            0.000348            1.023640   \n",
      "14               1.018133            0.000389            1.018821   \n",
      "15               1.013376            0.000418            1.014102   \n",
      "16               1.008726            0.000464            1.009501   \n",
      "17               1.004064            0.000449            1.004868   \n",
      "18               0.999584            0.000431            1.000426   \n",
      "19               0.995206            0.000443            0.996096   \n",
      "20               0.990870            0.000520            0.991806   \n",
      "21               0.986452            0.000508            0.987436   \n",
      "22               0.982110            0.000496            0.983136   \n",
      "23               0.977863            0.000484            0.978933   \n",
      "24               0.973657            0.000477            0.974768   \n",
      "25               0.969610            0.000549            0.970757   \n",
      "26               0.965524            0.000617            0.966707   \n",
      "27               0.961531            0.000615            0.962768   \n",
      "28               0.957613            0.000628            0.958881   \n",
      "29               0.953750            0.000633            0.955051   \n",
      "...                   ...                 ...                 ...   \n",
      "2470             0.509823            0.000734            0.582151   \n",
      "2471             0.509794            0.000734            0.582144   \n",
      "2472             0.509767            0.000732            0.582140   \n",
      "2473             0.509733            0.000732            0.582139   \n",
      "2474             0.509701            0.000733            0.582136   \n",
      "2475             0.509669            0.000735            0.582128   \n",
      "2476             0.509644            0.000735            0.582124   \n",
      "2477             0.509607            0.000736            0.582121   \n",
      "2478             0.509578            0.000737            0.582120   \n",
      "2479             0.509555            0.000735            0.582117   \n",
      "2480             0.509525            0.000738            0.582111   \n",
      "2481             0.509498            0.000740            0.582111   \n",
      "2482             0.509464            0.000737            0.582105   \n",
      "2483             0.509428            0.000740            0.582099   \n",
      "2484             0.509404            0.000736            0.582095   \n",
      "2485             0.509375            0.000731            0.582093   \n",
      "2486             0.509351            0.000732            0.582091   \n",
      "2487             0.509318            0.000729            0.582085   \n",
      "2488             0.509295            0.000728            0.582081   \n",
      "2489             0.509268            0.000731            0.582079   \n",
      "2490             0.509236            0.000731            0.582074   \n",
      "2491             0.509209            0.000732            0.582072   \n",
      "2492             0.509179            0.000730            0.582067   \n",
      "2493             0.509151            0.000731            0.582066   \n",
      "2494             0.509127            0.000734            0.582060   \n",
      "2495             0.509093            0.000736            0.582053   \n",
      "2496             0.509064            0.000738            0.582051   \n",
      "2497             0.509037            0.000735            0.582048   \n",
      "2498             0.509010            0.000730            0.582043   \n",
      "2499             0.508984            0.000727            0.582040   \n",
      "\n",
      "      test-mlogloss-std  \n",
      "0              0.000026  \n",
      "1              0.000131  \n",
      "2              0.000146  \n",
      "3              0.000191  \n",
      "4              0.000282  \n",
      "5              0.000311  \n",
      "6              0.000384  \n",
      "7              0.000457  \n",
      "8              0.000448  \n",
      "9              0.000377  \n",
      "10             0.000340  \n",
      "11             0.000331  \n",
      "12             0.000386  \n",
      "13             0.000503  \n",
      "14             0.000560  \n",
      "15             0.000603  \n",
      "16             0.000679  \n",
      "17             0.000695  \n",
      "18             0.000651  \n",
      "19             0.000635  \n",
      "20             0.000630  \n",
      "21             0.000630  \n",
      "22             0.000595  \n",
      "23             0.000565  \n",
      "24             0.000576  \n",
      "25             0.000645  \n",
      "26             0.000707  \n",
      "27             0.000719  \n",
      "28             0.000738  \n",
      "29             0.000761  \n",
      "...                 ...  \n",
      "2470           0.003234  \n",
      "2471           0.003230  \n",
      "2472           0.003228  \n",
      "2473           0.003228  \n",
      "2474           0.003229  \n",
      "2475           0.003226  \n",
      "2476           0.003227  \n",
      "2477           0.003231  \n",
      "2478           0.003230  \n",
      "2479           0.003234  \n",
      "2480           0.003237  \n",
      "2481           0.003237  \n",
      "2482           0.003239  \n",
      "2483           0.003236  \n",
      "2484           0.003237  \n",
      "2485           0.003239  \n",
      "2486           0.003238  \n",
      "2487           0.003239  \n",
      "2488           0.003240  \n",
      "2489           0.003237  \n",
      "2490           0.003237  \n",
      "2491           0.003238  \n",
      "2492           0.003238  \n",
      "2493           0.003238  \n",
      "2494           0.003239  \n",
      "2495           0.003239  \n",
      "2496           0.003240  \n",
      "2497           0.003238  \n",
      "2498           0.003237  \n",
      "2499           0.003238  \n",
      "\n",
      "[2500 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.5193595646083919\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8XHXZ9/HPNZOtaZKmadLSvQUKtCwWKJuCVBAFvGVRRFBQFOWRR0Rv9fZGcUEU9118VFTgdoMbRATZkUVEBJvKDgKltLSlS7qkTZo26/X88TtJpmkmmbSZnJnM9/16ndfMnHPmzHVmkvnOOb9zfsfcHREREYBE3AWIiEjuUCiIiEgPhYKIiPRQKIiISA+FgoiI9FAoiIhID4WCSAoz+5yZ/TLuOkTiolDIM2ZWYWbLzOy9KeMqzexVMzsjZdwCM7vNzDaZWaOZPWdmV5jZ+Gj6eWbWaWbN0bDUzC7Mcu0LzWxlNl9jKPqrx92/5u4fytLrLTOzN2dj2dkwUp9Xvr0vo51CIc+4ezPwf4AfmFldNPpbQL27/wHAzF4PPAj8HdjP3auBE4EO4HUpi/uHu1e4ewXwTuBbZnbwyKyJDIWZFcVdgxQId9eQhwNwLXAdsBDYAOyRMu1h4MeDPP884OE+4/4JvCfl8SnAs0AjIWTmpkybG41rjOY5JWXaycBzQBOwCvg0MBbYBnQBzdEwJc16/QS4PXr+Y8BeGbwf+wH3AhuBF4Azd6Ue4DLgt9HzZgEOfABYAWwCPgIcBjwVrfuVKa+zF3B/9HmsB34HVEfTfhO91rbotT6TwXu8DPjv6LVagaLo8apoXV4Aju/nvTgCWAMkU8adDjwV3T8cqAe2AGuB76V5TxcCK9NMGwf8GmgAlgOfBxLRtCTw3eg9eAW4KHofi9Isaxnw5jTTPgwsiT7XW7v/ZgADvg+si9bjaeCAdJ933P+v+TTEXoCGXfzgYDywOvrH+0DK+LFAJ7BwkOefR0ooRF90jcA+0eN9gK3ACUAx8Jnon7MkerwE+Fz0+LjoH3Df6LmrgWNS6jwkup/2SyaljmsJX6qHR1+CvwOuH+Q5Ywlf2h+InnNw9L7MG2o99B8KPwPKgLcA24E/AROBqdGX0rHR/HtH71cpUAc8BPwgZdk7fPkN9B6nzP8EMB0YA+wbreeUlPr6DUzgZeCElMc3ApdE9/8BnBvdrwCOTLOMtJ8XIRBuASqjOl4Ezo+mfYTwpTwter//wi6EQvR3tR44JHpPfww8FE17K7AYqCYExFxg8kCft4bMBu0+ylPuvonwC7Mc+GPKpPGE3YJrukeY2beidoWtZvb5lHmPjMY3EbYSfgO8FE17N3C7u9/r7u3AdwhfTK8HjiR8mXzD3dvc/X7gNuDs6LntwDwzq3L3Te7+ryGu3s3u/k937yCEwvxB5v8PYJm7X+PuHe7+OHAT8K5hqucr7r7d3e8hfIlf5+7r3H0V8DdCCOHuS6L3q9XdG4DvAccOsNyB3uNuP3L3Fe6+jRD2pdG6FLv7Mnd/Oc2yryP6PMyskvDr+bqU92NvM6t192Z3f3Qob4aZJYGzgM+6e5O7LyNsGZwbzXIm8EN3Xxn9nX5jKMtP8V7ganf/l7u3Ap8FjjKzWdE6VBK2EM3dn3f31Snrtzufd0FTKOQpMzuH8AvtL8A3UyZtIuymmNw9wt0/46Fd4WbCL+luj7p7tbtXAnsA+wNfi6ZNIewW6F5GF+FX6tRo2opoXLfl0TQI7RMnA8vN7K9mdtQQV29Nyv0WQgANZCZwRBRwjWbWSPhC2WOY6lmbcn9bP48rAMxskpldb2arzGwL8FugdoDlDvQed1uRMn0J8AnC1sy66LWmpFn274F3mFkp8A7gX+7e/VrnE7ZS/m1mi8zsPwaosT+1hC2b5SnjUj//Kal197k/FH3fn2bCVuTU6IfIlYRdjevM7Cozq4pm3d3Pu6ApFPKQmU0k7E/9MKHR+UwzOwbA3bcS9sO/YyjLdPe1hF/Xb49GvUb4su1+TSPsxlgVTZtuZql/PzOiabj7Inc/lbCL5U/ADd0vM5SahmAF8Nco4LqHCne/cITr+Vq0zAPdvQo4h7Bro1vf1xvoPe73Oe7+e3c/Onqes+MPgtT5niN8oZ4EvIcQEt3TXnL3swnvxzeBP5jZ2MxXk/WEX+MzU8b1fP6E3TfTUqZNH8KyU/V9f8YCE+j9O/uRux8KzCOE3H9F49N93pIBhUJ+uhL4k7s/EG0yfwb4RfSrkOjxB83skihAMLNpwOx0CzSzCYTGyGejUTcAbzOz482sGPgUobHzEULotACfMbNiM1tICJPrzazEzN5rZuOiXSJbCFsuEH5hTzCzccP0PnS7DdjHzM6N6ik2s8PMbO4I11NJaETebGZTib6kUqwF9kx5PNB7vBMz29fMjos+5+30NpSn83vg48AbCW0K3cs5x8zqoi2Txmh02uWYWVnqEM17A3BFdDj0TOCThC2j7vX6uJlNNbNqQuP4YIr7vE4RYXfXB8xsfrTOXwMec/dl0ed7RPS+bY3ej65BPm/JRNyNGhqGNgCnEX5BVfcZfz9wRcrjI4A7CP/0jcAzwBXAhGj6eYR91N1H3qwj/BNOTFnG6YQGw83AX4H9U6btH43bHM1zejS+BLiLsBtrC7AIODrleVcTdgE0kv7oo6+mPF7III3T0Xz7Eo5YaoiWfz+hLWJI9dB/Q3NRyvwrSWnEJ3wRfj7lPVkcvZ9PEL7kV6bMeyrwavRan87gPV7Gjg3TBxHafpoIR+Pc1t97mDL/DMIX4u19xv82+rybCT8CTkvz/IXR+vcd9ia0Xf02er9XAF+k9+ijIsKW7AbC0Uf/SdiysDSvs6yf1/hqNO0jhEbz7vWdFo0/nnBUVjO9R3pVDPZ5axh8sOgNFhHJCjM7CfiZu88cdGaJnXYficiwMrMxZnaymRVFu9G+RDjIQfKAthQkL0QN6Xf2N83DGdmSI8ysnLArbD9Cu8ftwMfdfUushUlGFAoiItJDu49ERKRH3nWyVVtb67NmzYq7DBGRvLJ48eL17l432Hx5FwqzZs2ivr4+7jJERPKKmS0ffC7tPhIRkRQKBRER6aFQEBGRHlkLBTO72szWmdkzaabvZ2b/MLNWM/t0tuoQEZHMZXNL4VrCJSDT2QhcTOhDXkREckDWQsHdHyJ88aebvs7dFxE6yhIRkRyQF20KZnaBmdWbWX1DQ0Pc5YiIjFp5EQrufpW7L3D3BXV1g5570a9VS5/j0Ru+w+bGDcNcnYjI6JEXoTAc1r3wGEc+9xU2rFwSdykiIjmrYEKhpKIGgG1btKUgIpJO1rq5MLPrCFduqjWzlYQ+1YsB3P1nZrYHUA9UES6j9wlgXra61y2tmgBAW3Patm8RkYKXtVDwcGHwgaavYceLe2dVeVUtAO0KBRGRtApm91HF+BAKXS2bYq5ERCR3FU4oVI6nyw3f1hh3KSIiOatgQiGRTNJk5VirQkFEJJ2CCQWAZqugqFWXiRURSaegQqElWUlx++a4yxARyVkFFQqtyUpKO5riLkNEJGcVVCi0FVdR3qlQEBFJp6BCoaNkHGO9Oe4yRERyVkGFQldZNZXejHd1xV2KiEhOKqhQeHydU2KdbN2qXUgiIv0pqFBYsN9sAJoa18dciYhIbiqoUCha9iAALZsVCiIi/SmoUCg54kMAbNuiTvFERPpTUKFQWhmuqdCqnlJFRPpVUKEwtjr0lNqxVT2lioj0p6BCoaJa3WeLiAyksEKhqoYuN55fujzuUkREclJBhYIlkmyxCuZWd8RdiohITiqoUABoSlRRvF27j0RE+lNwobA1OY7Sdl1oR0SkPwUXCttLxlPeoVAQEelPwYXC1qYtVHYqFERE+lNwoVA07WDG0UxXp3pKFRHpq+BCwcZOoNQ62NKkrQURkb6yFgpmdrWZrTOzZ9JMNzP7kZktMbOnzOyQbNWS6sEVnQBs2bB2JF5ORCSvZHNL4VrgxAGmnwTMiYYLgJ9msZbeFz3iAAC2NioURET6yloouPtDwEA9z50K/NqDR4FqM5ucrXq6jRk3EYBtjQ3ZfikRkbwTZ5vCVGBFyuOV0bisGjs+hMKd/3w22y8lIpJ38qKh2cwuMLN6M6tvaNi9X/jjJuwBwMLpebHqIiIjKs5vxlXA9JTH06JxO3H3q9x9gbsvqKur260XLa+socMT0LJht5YjIjIaxRkKtwLvi45COhLY7O6rs/6qZmyikjWr+80fEZGCVpStBZvZdcBCoNbMVgJfAooB3P1nwB3AycASoAX4QLZq6Wtr0Thmlm4fqZcTEckbWQsFdz97kOkOfDRbrz+Q7UXjKFOneCIiOynI1tZVreWUtqn7bBGRvgoyFMbVTWECm+MuQ0Qk5xRkKPjYiVTTzNaWbXGXIiKSUwoyFJJrnyZhzqaG1+IuRUQkpxRkKNxqCwHYskGhICKSqiBD4X0nHAZAi0JBRGQHBRkKVbVTAGhd/PuYKxERyS0FGQrVtaHfvS0tOoFNRCRVQYZC0ZhKmr2MxqLauEsREckpBRkKAFuS49lzTEvcZYiI5JSCDYWmrlLGbF0x+IwiIgWkYEOh1cZQ6c1xlyEiklMKNhReshmMo4nQL5+IiEABh0Jjcjw11kyTuroQEelRsKFw6Lx9Adi4ThfbERHpVrChULr0PgA2N6yMuRIRkdxRsKFQeeLnAGj565UxVyIikjsKNhRqJs8E4JG2PWOuREQkdxRsKJRXT6bDE9R0ro+7FBGRnFGwoUAiyTrGM6FrQ9yViIjkjKK4C4hTS+lEZnhj3GWIiOSMgg6FbR3OuM61cZchIpIzCnf3EdCRKKOORto6uuIuRUQkJxR0KFhRCRW2nYb1DXGXIiKSEwo6FIoPfjcAjWuXx1yJiEhuyGoomNmJZvaCmS0xs0v6mT7TzO4zs6fM7EEzm5bNevqqqJsOwP/c9chIvqyISM7KWiiYWRL4CXASMA8428zm9ZntO8Cv3f0g4HLg69mqpz81k2YDUK1zFUREgOxuKRwOLHH3pe7eBlwPnNpnnnnA/dH9B/qZnlUVdWHDpKZT5yqIiEB2Q2EqkHpps5XRuFRPAu+I7p8OVJrZhCzWtAMrKafRK5jQqYZmERGIv6H508CxZvY4cCywCujsO5OZXWBm9WZW39AwvF/gq62Wia7dRyIikN1QWAVMT3k8LRrXw91fc/d3uPvBwKXRuJ1OMXb3q9x9gbsvqKurG9Yi19pEpvi6YV2miEi+ymYoLALmmNlsMysBzgJuTZ3BzGrNrLuGzwJXZ7Gefq1LTGKqrad5e/tIv7SISM7JWii4ewdwEXA38Dxwg7s/a2aXm9kp0WwLgRfM7EVgEnBFtupJZ13RJMqtlddeWzH4zCIio1xW2xTc/Q5338fd93L3K6JxX3T3W6P7f3D3OdE8H3L31mzW05+TjzkSgB/fdN9Iv7SISM6Ju6E5djVT9wJgQvuamCsREYlfwYfCuMkhFOo6FQoiIgUfClY2jkavoE5daIuIFPb1FLo1lk5mtq7AJiKiLQWAlvKpTGhfg7vHXYqISKwUCkBH1XSm0MCmrW1xlyIiEiuFAvDXdeWUWTurVy2LuxQRkVgpFIDTy54A4Ge33D/InCIio9ugoWBme5lZaXR/oZldbGbV2S9t5NSd9WMA3th8T8yViIjEK5MthZuATjPbG7iK0Mnd77Na1QgrrduTDk9QyoifUC0iklMyCYWuqB+j04Efu/t/AZOzW9YISxazvngPasos7kpERGKVSSi0m9nZwPuB26JxxdkrKR5bymcyoXWlDksVkYKWSSh8ADgKuMLdXzGz2cBvslvWyHuyZQIzWU1D0/a4SxERic2goeDuz7n7xe5+nZmNByrd/ZsjUNuImtG5nHJrZcWrS+MuRUQkNpkcffSgmVWZWQ3wL+AXZva97Jc2smbXlAHwm9vUhbaIFK5Mdh+Nc/ctwDuAX7v7EcCbs1vWyKs9J1z0bVL7qkHmFBEZvTIJhSIzmwycSW9D86iTGDeNVi9i/7an4i5FRCQ2mYTC5YRLar7s7ovMbE/gpeyWFYNEgjVMYIJtjrsSEZHYDNp1trvfCNyY8ngp8M5sFhWXRiqZZWvY3NLOuPJRd9StiMigMmlonmZmN5vZumi4ycymjURxI63DSphqG1iyQu0KIlKYMtl9dA1wKzAlGv4cjRt1ZkVHIF19850xVyIiEo9MQqHO3a9x945ouBaoy3Jdsag5N2TdlPZl8RYiIhKTTEJhg5mdY2bJaDgHGJXXrrRxM9jmJRygI5BEpEBlEgofJByOugZYDZwBnJfFmuKTSLCGGiaxUX0giUhByqSbi+Xufoq717n7RHc/jVF69BFAR0k1eyVeY80W9YEkIoVnV6+89slMZjKzE83sBTNbYmaX9DN9hpk9YGaPm9lTZnbyLtYzbMpKS6izLVz8i3vjLkVEZMTtaigMeuEBM0sCPwFOAuYBZ5vZvD6zfR64wd0PBs4C/t8u1jNsJr79MgDeuuXGgWcUERmFdjUUMtnhfjiwxN2XunsbcD1waj/LqYrujwNe28V6hk3p9IMBKPNtMVciIjLy0p7RbGZN9P/lb8CYDJY9FViR8nglcESfeS4D7jGzjwFjSdPRnpldAFwAMGPGjAxeejeU17DOq5kYNTab6WpsIlI40m4puHulu1f1M1S6+6DdY2TobOBad58GnAz8xsx2qsndr3L3Be6+oK4u+6dILPb9mGuvsnKTthZEpLDs6u6jTKwCpqc8nhaNS3U+cAOAu/8DKANqs1hTRuZXNTE90cDHr9a1FUSksGQzFBYBc8xstpmVEBqSb+0zz6vA8QBmNpcQCg1ZrCkjE075CqDGZhEpPFkLBXfvAC4idLv9POEoo2fN7HIzOyWa7VPAh83sSeA64DzPgbPGSqaFxuZtibExVyIiMrKGq22gX+5+B3BHn3FfTLn/HPCGbNawS8prWO9V7Nf1Iu2dXRQns7lBJSKSOzLpOrvJzLb0GVZE3WnvORJFxmGDV/G6xFKeW6WL7ohI4cjkJ/APgP8iHGI6Dfg08HvCeQdXZ6+0eE0eX8Fk28jHrxq1VyAVEdlJJqFwirv/3N2b3H2Lu18FvNXd/xcYn+X6YlP17p8DcDo6AklECkcmodBiZmeaWSIazgS6e4uLvVE4ayYdwHYvZiKN6jFVRApGJqHwXuBcYF00nAucY2ZjCEcXjU7JIjZ6JQckXuHlhua4qxERGRGZdJ291N3f7u610fB2d1/i7tvc/eGRKDIuY6vGM9de5cwfqcdUESkMmRx9NC060mhdNNxkZtNGori4jSsfQ5F18U7/S9yliIiMiEx2H11DOBN5SjT8ORo3+n34Pto9yVRroK2jK+5qRESyLpNQqHP3a9y9IxquBbLfK10uKC5jrU3giMTz/OvVTXFXIyKSdZmEwgYzO8fMktFwDrAh24XlitqqSuYmVvDJX90ddykiIlmXSSh8EDgTWAOsBs4AzstiTTml7KxfAfA2G9Vt6iIiQGZHHy1391Pcvc7dJ7r7acA7R6C23LDHQbR4KXNYycpNLXFXIyKSVbva09snh7WKXJZI0jWmhmOTT/Lm7z4QdzUiIlm1q6FQUNeorBhbySRrZG7XkrhLERHJql0NhcLq9+H8u+ly482JehqaWuOuRkQka9KGQpous7eYWRPhfIXCUV7DRhvH8YnHOfqb98ddjYhI1qS9yI67V45kIbluQvV4ahtfYVL76rhLERHJGl1SLEN2Xri89NsSj7Fs/daYqxERyQ6FQqaqZ9BWXM0pyUd403cejLsaEZGsUCgMQUlFDXMTr7KXraSrq7Da2kWkMCgUhiI6CumU5COc9MO/xV2NiMiwUygMRcVEGFPNGcmHeGnt5rirEREZdgqFIUpUTGSKbeToxNNqcBaRUUehMFQf+RtdiWLOTt7PQjU4i8goo1AYqqJSEma8JbGYWjazcWtb3BWJiAybrIaCmZ1oZi+Y2RIzu6Sf6d83syei4UUza8xmPcPmwkdIWhfnFN3LIV/R9ZtFZPTIWiiYWRL4CXASMA8428zmpc7j7v/p7vPdfT7wY+CP2apnWNXOgUQRH0reQRmt2loQkVEjm1sKhwNL3H2pu7cB1wOnDjD/2cB1WaxneL3/NipsO2cmH+RQbS2IyCiRzVCYCqxIebwyGrcTM5sJzAb67W3OzC4ws3ozq29oaBj2QnfJjCPBklxcdDMJOnlFRyKJyCiQKw3NZwF/cPfO/ia6+1XuvsDdF9TV1Y1waWmYQe0+1NoW3pl4SF1fiMiokM1QWAVMT3k8LRrXn7PIp11H3S58BCzBpcW/o5gO5n7hzrgrEhHZLdkMhUXAHDObbWYlhC/+W/vOZGb7AeOBf2SxluxIJKBuLuOshQuTt7CtvYvGFjU6i0j+yloouHsHcBFwN/A8cIO7P2tml5vZKSmzngVc7+752cPchX8HS/Kxoj9RxVbmX65GZxHJX5Zv38ULFizw+vr6uMvY0WtPwFXH8njnXpze/hUMeOUbb4u7KhGRHma22N0XDDZfrjQ057cp8wFjfvJl9rdlOLC0oTnuqkREhkyhMFz++xUM+GPJFymmg+O++1eatrfHXZWIyJAoFIbLmPHw7t9Sah3cU/JfABx42T10dHbFXJiISOYUCsNp7tsBY3ZiLYfaiwDsfemdukqbiOQNhcJwu+RVAG4qvYxxhHaFPT93B50KBhHJAwqF4VZWBef/BYAnxlxIknCS9l6fu4N27UoSkRynUMiG6YfBhDmYd/Ly2PN7Rs+59E6aWztiLExEZGAKhWz5WD0kS6CzjWWTL+0ZfcCX7tZlPEUkZykUsunSNeF20yssm/olxhSHt3vhdx5kz0tuj7EwEZH+KRSyKZGEz68L9ze8xPPvauJvn3kTAF3ArEtuZ/M2ncsgIrlDoZBtRaXw2ahz2D9+mOmv3MiLXz2pZ/LrvnyPelcVkZyhUBgJpRW9u5L+fDElPz6IZV87ifKSJADb2ruYdcntLN+gtgYRiZc6xBtJHW3wtSnQ1Q6JIvjv5WxPjGG/L9y1w2z//NzxTKwqi6lIERmN1CFeLioqgS80wPg9oasDvj6VsuYVLPvG23jsc8f3zHb41+5j1iW3s2JjS4zFikgh0pZCXF6+H35zerh/xtVwwDsBWL5hK8d++8EdZr35/76eg2eMH+ECRWQ0yXRLQaEQpw0vw48PCffL6+Bji0LHesDqzds46uv37/SUpy97C5VlxSNZpYiMAgqFfNHZAQ9/Dx64Ijx+17Uw7zQwA6C5tYMDvnT3Tk/bd1IFf/7YMZQUaQ+giAxOoZBvXnscrloY7lsSPvoY1M7pmezuPLlyM6f95O87PXVMcYLFXziB8pKiESpWRPKNQiEfdXZA/dVwZ7geA0ddBMd8CsprdpittaOT+V++h23t/Xewd/P/fT0HTh1HUVJbESISKBTyWfM6+OkbYGt0NvRxX4AjL4SSsTvN2trRyalX/p1/r2lKu7j7P3Uss2vHYtEuKREpPAqF0WDtc/DzY8LhqxDC4bAPwZjqtE/ZuLWNQ75y74CL/dX7F3DQtGrqKkuHs1oRyWEKhdHk1cfg2pN7w+ENnwjhUD19wKe5O8s3tLDwOw8O+hL77VHJtR84nElVpdqiEBmFFAqj0eqnwrkNLevD431OhAXnw97Hh873MtDS1sGhX/0L29o6M5r/5+ceyt4TK5hZU642CpE8plAYzRpfhX/9Gh76du+4474Ah7wPKiYOeXGbtrZx9i8eHbBdoj9/vuho9qwby9hSHfUkkusUCoWgsx3+fTvc+P7ecWPGw9u+C/ueDMVjdmvxTdvbOfCye3b5+X/4yFFMrh7DxMpSirWVIRKrnAgFMzsR+CGQBH7p7t/oZ54zgcsAB5509/cMtEyFQhrrX4InfgcPf7933P6nw7xTYc5b+j1yaVe1dXSxfMNWTvj+Q7u9rJsufD2Tx5UxsbJUu6dEsij2UDCzJPAicAKwElgEnO3uz6XMMwe4ATjO3TeZ2UR3XzfQchUKg+jqhGUPw3N/Cuc8dJv79nCm9Jy3QFlV1l7e3Wlq7eCg3djCSOeWj76BmrEl1IwtobwkqQZxkSHIhVA4CrjM3d8aPf4sgLt/PWWebwEvuvsvM12uQmEIujph+SPw3C2w6Be942cdA3NOgL1PgIlze7rUGCnuzpbtHbzuy8MfHH3d9YljqBlbwvjyEu3CkoKWC6FwBnCiu38oenwucIS7X5Qyz58IWxNvIOxiuszd7+pnWRcAFwDMmDHj0OXLl2el5lGtqwtWPAYv3gV//8GO0w49L2xBzH4jlFbGUl5/urc6Nja3ZXRY7XBLALddfAyVZUVUlRVTUVZEMqGtE8lP+RIKtwHtwJnANOAh4EB3b0y3XG0pDJPNq2DJvfDSvfDv23rHl1TC4R+CmUfDjCNyKiQy0d7ZxaaWNg6/4r64S9lJArjloqMZU5JkTEmS8uJwW1qU0K4wybpcCIVMdh/9DHjM3a+JHt8HXOLui9ItV6GQBR1tsOJRWPogPPwD8JRzGKYugFlHh2HGkXkXEkPR2tFJ0/YOzr92EU+u3Bx3Obvlto8dTWlRgrLiEDqlRUlKixMKoAKWC6FQRNg1dDywitDQ/B53fzZlnhMJjc/vN7Na4HFgvrtvSLdchcIIaNsKK/4ZGqwf/n6fkDgUZr4htEtMP6zn+g/Sq6Ozi23tnWxr78zJLZY43PiRoyhKGMXJBEXJcFucCPeLkkZxIkFxUaJnHu2mG36xh0JUxMnADwjtBVe7+xVmdjlQ7+63WvjJ8l3gRKATuMLdrx9omQqFGLS1wMooJJY9DK/+o3fahDkw/XCYtgCmHQZ1cyGpk9myqTt02jq62N7RRWt7J8d9969xl1VQ7vz4MRQnu0MuQXHCKIoCryQZwi2ZsJzaKsuJUMgGhUIOaGuBVfVha2JlPbx4547TZx4NU+bD5NeFYcLeGXfDIbmvq8vp6HLaO7vo6HTau7pY8NW/xF1WQTDglW+8bdeem2Eo6CedDF1JeThSafYbw2N32LQsBMTKRSEw/nHljs+ZfkRvSEyeD3X7QlKXFc1HiYRRkrAdrvq3bBe/qOLiHoJSJGdSAAAMaUlEQVRtzqV3Dj5zgdGWgmRHZwesfxFWP9k7vPrIjvNMOSQKiYNg4v7hnIksnlgnUsi0pSDxShbBpHlhmH92GNfVBRuXwuoneoNi8TV9nlcKe70JJu0PE+eF25q9oKhk5NdBpAApFGTkJBJQu3cYDjwjjHOHzSvCBYXWPgPrngv3X+xzDuOEvaF233Dd6tp9wu6n2jlQNm7k10NkFFMoSLzMoHpGGPY9sXd8R2vY/bT2uXC7/oXQ6d8Lt+/4/IpJISRSg6J2H6iaOuLdd4iMBgoFyU1FpbDHgWFI1dkRGrVTg2L9i1D/q52X0d2g3R0UE+bA+FmhoVxE+qVQkPySLOrdBcXJvePdoXndjmHR8AI8dQOhV/YUlVOgZk+omR0Ne4Zh/Gw1dEvBUyjI6GAGlZPCMPuYHae1NsOGl2DDy7DxldDYvXEpPP5bdgqMRDFMPaQ3KGr2hOqZMH4mjK3TLikZ9RQKMvqVVsCUg8PQV2tT2B3VHRQbl4bgePJ6dgoMCLuhqmeGNpDxM3vbQ6pnQXmNQkPynkJBCltpZf9tFwDt20JgbFoerovduDwMm5aHHmb7KqlICYmUwOgOD/UTJXlAoSCSTvGYcELdxLn9T9++OQqLV3cMjqUPQnvLzvNbMpx7scNWRnQ7blo4vFZbGhIzhYLIriobl34rwx22bUrZwkgJjo0v73xobbfafUNAjJsKVd23U8O4qqk6ckqyTqEgkg1moY2hvCZ0DtiXO7RsiIJiWbjo0ZZVsHlluH35fvpt0wCYdGAUFlOiYWrKMEXBIbtFoSASBzMYWxuGaYf2P09HaxQUfQJjczT0Pes71aQD+oTGFKic3HurXVWShkJBJFcVlfYeFptO+zbY8loIi9Tb7iB56Z70z63ZKyUsJofzN6omQ8Ue4dDeij2guGz410tymkJBJJ8Vj4EJe4Uhnfbt0LQ6DFtei25XQ9Nr4XbFo6GtI53afXtDoud2j9DFSPdtaaW2PEYJhYLIaFdc1nv2djrdbRxbXoPmtdC0BprXQNPa3tunbyRtOweEM8L7hsUOt3voXI48oFAQkR3bOAbiDtsbdwyLvrfP3wrelX4ZVdP62fKIbismhgAZO1GXdY2J3nURyZxZOAlvzHiYuN/A87ZtjbY41vZ/u+mV9IfmQuhWZIfAqAvjxk6MAqwuhMiYGgXIMNI7KSLZUTJ28PYOgI62EBTpdls1r4El9zHgrqtEUegFNzUsuu+PnRjd1obxJWOHdTVHG4WCiMSrqASqp4dhIN0nBG5dD1sbYOu63vvN66Jx68OV/TYuTb8cS4STAfuGxdi6nYfyGkgkh3d9c5xCQUTyQ+oJgXX7DD5/+/YoKKKw2JoSHN0hsnklvHjnwMvpLywqUh+nbJWMghMHFQoiMjoVl2W2BQLh+uHbG1O2OPoMzdHtqsWhLSQdS4bX66/tY4fdWbWhXSYHt0IUCiIiiUTvVgiDNKADtLVAy/resOhvd9amZbByURifTvkEKI+O+iqfEN0O8DhZPFxrnJZCQURkqErKoSTqGn0wXZ1RW0ifto+W9Sm3G6Dh37BsPWzbmH5ZVdPgk88O33r0I6uhYGYnAj8EksAv3f0bfaafB3wbWBWNutLdf5nNmkRERlQi2XsOSLpu2FN1dULLxj6hsT6cXDjtsKyXm7VQMLMk8BPgBGAlsMjMbnX35/rM+r/uflG26hARySuJZGjIrqiL5+WzuOzDgSXuvtTd24DrgVOz+HoiIrKbshkKU4EVKY9XRuP6eqeZPWVmfzCzfg8TMLMLzKzezOobGhqyUauIiJDdUMjEn4FZ7n4QcC/wP/3N5O5XufsCd19QVxfPJpWISCHIZiisAlJ/+U+jt0EZAHff4O6t0cNfAmmuNiIiIiMhm6GwCJhjZrPNrAQ4C7g1dQYzm5zy8BTg+SzWIyIig8ja0Ufu3mFmFwF3Ew5JvdrdnzWzy4F6d78VuNjMTgE6gI3AedmqR0REBmfuA/Q8mIMWLFjg9fX1cZchIpJXzGyxuy8YbL64G5pFRCSH5N2Wgpk1AMt38em1wPphLCcfaJ0Lg9a5MOzOOs9090EP38y7UNgdZlafyebTaKJ1Lgxa58IwEuus3UciItJDoSAiIj0KLRSuiruAGGidC4PWuTBkfZ0Lqk1BREQGVmhbCiIiMgCFgoiI9CiYUDCzE83sBTNbYmaXxF3PcDKzZWb2tJk9YWb10bgaM7vXzF6KbsdH483MfhS9D0+Z2SHxVp8ZM7vazNaZ2TMp44a8jmb2/mj+l8zs/XGsS6bSrPNlZrYq+qyfMLOTU6Z9NlrnF8zsrSnj8+Jv38ymm9kDZvacmT1rZh+Pxo/az3mAdY7vc3b3UT8Q+l56GdgTKAGeBObFXdcwrt8yoLbPuG8Bl0T3LwG+Gd0/GbgTMOBI4LG4689wHd8IHAI8s6vrCNQAS6Pb8dH98XGv2xDX+TLg0/3MOy/6uy4FZkd/78l8+tsHJgOHRPcrgRej9Rq1n/MA6xzb51woWwqFeBW4U+m9PsX/AKeljP+1B48C1X16q81J7v4QodPEVENdx7cC97r7RnffRLiGx4nZr37XpFnndE4Frnf3Vnd/BVhC+LvPm799d1/t7v+K7jcRek2eyij+nAdY53Sy/jkXSihkehW4fOXAPWa22MwuiMZNcvfV0f01wKTo/mh6L4a6jqNl3S+Kdpdc3b0rhVG2zmY2CzgYeIwC+Zz7rDPE9DkXSiiMdke7+yHAScBHzeyNqRM9bHeO6mOPC2EdIz8F9gLmA6uB78ZbzvAzswrgJuAT7r4lddpo/Zz7WefYPudCCYVBrwKXz9x9VXS7DriZsCm5tnu3UHS7Lpp9NL0XQ13HvF93d1/r7p3u3gX8gvBZwyhZZzMrJnw5/s7d/xiNHtWfc3/rHOfnXCihMOhV4PKVmY01s8ru+8BbgGcI69d91MX7gVui+7cC74uO3DgS2JyyaZ5vhrqOdwNvMbPx0eb4W6JxeaNP+8/phM8awjqfZWalZjYbmAP8kzz62zczA34FPO/u30uZNGo/53TrHOvnHHfr+0gNhCMVXiS00F8adz3DuF57Eo40eBJ4tnvdgAnAfcBLwF+Ammi8AT+J3oengQVxr0OG63kdYTO6nbC/9PxdWUfgg4TGuSXAB+Jer11Y599E6/RU9E8/OWX+S6N1fgE4KWV8XvztA0cTdg09BTwRDSeP5s95gHWO7XNWNxciItKjUHYfiYhIBhQKIiLSQ6EgIiI9FAoiItJDoSAiIj0UCiIi0kOhIJIBM5vfp/viU4arG2oz+4SZlQ/HskR2l85TEMmAmZ1HODnqoiwse1m07PVDeE7S3TuHuxYRbSnIqGJms8zseTP7RXTRknvMbEyaefcys7ui3mX/Zmb7RePfZWbPmNmTZvZQ1G3A5cC7owuevNvMzjOzK6P5rzWzn5rZo2a21MwWRj1bPm9m16a83k/NrD6q68vRuIuBKcADZvZANO5sCxdNesbMvpny/GYz+66ZPQkcZWbfsHBxlqfM7DvZeUel4MR9mrcGDcM5ALOADmB+9PgG4Jw0894HzInuHwHcH91/Gpga3a+Obs8Drkx5bs9j4FpC//VG6MN+C3Ag4UfX4pRaurtnSAIPAgdFj5cRXSSJEBCvAnVAEXA/cFo0zYEzo/sTCN0cWGqdGjTs7qAtBRmNXnH3J6L7iwlBsYOoq+LXAzea2RPAzwlXwQL4O3CtmX2Y8AWeiT+7uxMCZa27P+2hh8tnU17/TDP7F/A4sD/hKlp9HQY86O4N7t4B/I5wBTaATkJvmgCbge3Ar8zsHUBLhnWKDKgo7gJEsqA15X4n0N/uowTQ6O7z+05w94+Y2RHA24DFZnboEF6zq8/rdwFFUY+WnwYOc/dN0W6lsgyWm2q7R+0I7t5hZocDxwNnABcBxw1xeSI70ZaCFCQPFzJ5xczeBT0XgX9ddH8vd3/M3b8INBD6qW8iXEN3V1UBW4HNZjaJcEGkbqnL/idwrJnVmlkSOBv4a9+FRVs649z9DuA/gdftRm0iPbSlIIXsvcBPzezzQDGhXeBJ4NtmNofQRnBfNO5V4JJoV9PXh/pC7v6kmT0O/Jtw2cS/p0y+CrjLzF5z9zdFh7o+EL3+7e5+y85LpBK4xczKovk+OdSaRPqjQ1JFRKSHdh+JiEgP7T6SUc/MfgK8oc/oH7r7NXHUI5LLtPtIRER6aPeRiIj0UCiIiEgPhYKIiPRQKIiISI//DybkACRBX1YwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.01,\n",
    "        n_estimators=2500,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha = 1.5,\n",
    "        reg_lambda = 0.5,    \n",
    "        seed=3)\n",
    "\n",
    "\n",
    "modelfit(xgb5_1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators is :  2500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zhangwt/.local/lib/python3.6/site-packages/ipykernel_launcher.py:2: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# n_estimators\n",
    "cvresult = pd.DataFrame.from_csv('5_nestimators.csv')\n",
    "#cvresult.shape\n",
    "#cvresult.head\n",
    "print(\"n_estimators is : \", cvresult.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train-mlogloss-mean is : 0.5651147999999999\n",
      "test-mlogloss-mean is : 0.598666\n"
     ]
    }
   ],
   "source": [
    "# train-mlogloss-mean\n",
    "print(\"train-mlogloss-mean is :\", cvresult['train-mlogloss-mean'][1007])\n",
    "print(\"test-mlogloss-mean is :\", cvresult['test-mlogloss-mean'][1007])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 学习率取0.01时，n_estimators为2500，仍未收敛，下一步增大学习率至0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
